import requests
from lxml import etree
from pandas import DataFrame

class LianJiaSpider():
    def __init__(self):
        self.url = "https://bj.lianjia.com/ershoufang/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
        }

    def send_request(self, url):
        response = requests.get(url, headers=self.headers)
        return response

    def parse_content(self, response):

        # with open("lj.html", "w", encoding="utf8") as f:
        #     f.write(response.text)
        html = etree.HTML(response.text)
        li_list = html.xpath('//ul[@class="sellListContent"]//li')
        d = {"title": [],
             "follow_info": [],
             "total_rice": [],
             "unit_price": [],
             "type": [],
             "area": [],
             "dir": [],
             "ren": [],
             "floor": [],
             "year": [],
             "other": [],
             }
        for li in li_list:
            title = "".join(li.xpath('.//div[@class="title"]/a/text()'))
            house_info = "".join(li.xpath('.//div[@class="houseInfo"]/text()'))
            follow_info = "".join(li.xpath('.//div[@class="followInfo"]/text()'))
            total_rice = "".join(
                li.xpath('.//div[@class="priceInfo"]/div[@class="totalPrice totalPrice2"]/span/text()'))
            unit_price = "".join(li.xpath('.//div[@class="priceInfo"]/div[@class="unitPrice"]/span/text()'))

            if title and follow_info and house_info and total_rice and unit_price:
                # 处理数据
                l = house_info.split("|")
                if len(l) == 6:
                    l.insert(-1, "1970年")
                type = l[0]  # 户型
                area = l[1]
                dir = l[2]  # 朝向
                ren = l[3]  # 装修
                floor = l[4]  # 楼层
                year = l[5]  # 年份
                other = l[6]
                d["title"].append(title)
                d["follow_info"].append(follow_info)
                d["total_rice"].append(total_rice)
                d["unit_price"].append(unit_price)
                d["type"].append(type)
                d["area"].append(area)
                d["dir"].append(dir)
                d["ren"].append(ren)
                d["floor"].append(floor)
                d["year"].append(year)
                d["other"].append(other)
        self.save_content(d)

    def save_content(self,d):
        df = DataFrame(data=d)
        df.to_excel("house.xlsx",index=False)

    def start(self):
        response = self.send_request(self.url)
        if response.status_code == 200:
            self.parse_content(response)


if __name__ == '__main__':
    ljs = LianJiaSpider()
    ljs.start()

